{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Evaluation \n",
    "\n",
    "This notebook contains the code for the evaluation of the scoring performance, consistency and the automatic evaluations for the helpfulness of the feedback as well as the comparison to the manual annotation study.\n",
    "\n",
    "#### Imports and General Variables"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b3d55c82a2ba4ac"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import json\n",
    "import multiprocessing\n",
    "import os\n",
    "import random\n",
    "\n",
    "import krippendorff\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datasets import load_from_disk\n",
    "from langchain.output_parsers import StructuredOutputParser, ResponseSchema\n",
    "from langchain.prompts import PromptTemplate\n",
    "from numpy import mean, std\n",
    "from scipy.stats import kendalltau\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "from tqdm import tqdm\n",
    "\n",
    "from analysis import calculate_similarity_for_cluster_part, create_embedding, compare_texts\n",
    "from essay_meta_data import essay_set_descriptions\n",
    "from evaluation_helpers import qwk\n",
    "from helpfulness_example_dataset import HELPFUL, NOT_HELPFUL, SEMI_HELPFUL\n",
    "from llm_functions import collect_llm_output\n",
    "from llm_functions import load_model\n",
    "\n",
    "df = pd.read_excel(\"../data/training_set_rel3.xlsx\", index_col=\"essay_id\")\n",
    "df = df.drop(10534)  # this essay is not annotated \n",
    "\n",
    "os.makedirs(\"../data/results/\", exist_ok=True)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "eb244c0ea43b734d"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Scoring Performance Evaluation \n",
    "\n",
    "For the scoring performance evaluation, the QWK scores for the different prompts have to be evaluated. One experiment can be done via: \n",
    "```bash \n",
    "python main.py --logging_data_path=\"./data/qwk/template1_1/one_shot_prompt.json\" --prompt=one_shot_prompt --dataset_split=dev --model_size=\"7b\" --model=\"mistral\" --prompt-template=1 --instruction-variant=1 --setting=one-shot --full-rubric\n",
    "```\n",
    "This command will generate the QWK scores for the one-shot prompt using the basic template and the original formulation variant. The variable `experiments` then has to be filled with all the experiments that have been conducted.\n",
    "\n",
    "The result of the following cell is a pandas dataframe that contains the QWK scores for each essay set and the average QWK score over all essay sets. The dataframe also contains the number of incorrect predictions for each essay set.\n",
    "Using this dataframe, the results can be visualized and analyzed."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "9ae8a3c8fc1c6d93"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        \"data_path\": \"./data/qwk/template1_1/one_shot_prompt.json\",\n",
    "        \"prompt\": \"one_shot_prompt\",\n",
    "        \"template\": 1,\n",
    "        \"variant\": 1,\n",
    "        \"model\": \"mistral\",\n",
    "    }\n",
    "]\n",
    "\n",
    "df_data = []\n",
    "experiment_results = {}\n",
    "for i, experiment in tqdm(enumerate(experiments)):\n",
    "    with (open(experiment[\"data_path\"]), \"r\") as f:\n",
    "        results = json.load(f)\n",
    "    experiments[i][\"results\"] = results\n",
    "\n",
    "    df[f\"{experiment}_predicted_score\"] = 0\n",
    "\n",
    "    for fold in results:\n",
    "        for j in range(len(results[fold])):\n",
    "            essay_id = results[fold][j][\"id\"]\n",
    "            output = results[fold][j][\"parsed_output\"]\n",
    "            essay_set = df.loc[essay_id]['essay_set']\n",
    "            meta_data = essay_set_descriptions[essay_set - 1]\n",
    "            try:\n",
    "                score = meta_data['full_score_fn'](output, output)\n",
    "            except Exception as e:\n",
    "                score = -1  # mark the score as incorrect to filter it out later\n",
    "\n",
    "            df.loc[essay_id, f\"{experiment}_predicted_score\"] = score\n",
    "\n",
    "    wrong_predictions = 0\n",
    "    for essay_set in range(1, 9):\n",
    "        meta_data = essay_set_descriptions[essay_set - 1]\n",
    "        score_range = meta_data['score_ranges'][0]\n",
    "        filtered_df = df[df[\"essay_set\"] == essay_set]\n",
    "        incorrect_predictions = filtered_df[(filtered_df[f\"{experiment}_predicted_score\"] < score_range[0]) | (\n",
    "                filtered_df[f\"{experiment}_predicted_score\"] > score_range[1])]\n",
    "        # change the scores to -1 for the incorrect predictions in the original dataframe\n",
    "        df.loc[incorrect_predictions.index, f\"{experiment}_predicted_score\"] = -1\n",
    "        wrong_predictions += len(incorrect_predictions)\n",
    "\n",
    "    experiments[i][\"wrong_predictions\"] = wrong_predictions\n",
    "\n",
    "    # calculate the QWK score for each essay set\n",
    "    qwks = [[] for _ in range(8)]\n",
    "    for fold in results:\n",
    "        essay_set = fold.split(\"_\")[-1]\n",
    "        fold_index = int(fold.split(\"_\")[1])\n",
    "        filtered_df = df[df[\"essay_set\"] == int(essay_set) + 1]\n",
    "\n",
    "        # get the true and predicted scores but filter out the incorrect predictions with -1\n",
    "        clean_true_scores = filtered_df[filtered_df[f\"{experiment}_predicted_score\"] != -1][\"domain1_score\"].filter(\n",
    "            items=[essay['id'] for essay in experiment_results[experiment][fold]])\n",
    "        clean_predicted_scores = filtered_df[filtered_df[f\"{experiment}_predicted_score\"] != -1][\n",
    "            f\"{experiment}_predicted_score\"].filter(\n",
    "            items=[essay['id'] for essay in experiment_results[experiment][fold]])\n",
    "        qwks[int(essay_set)].append(qwk(clean_predicted_scores, clean_true_scores))\n",
    "\n",
    "    # take the average over all folds\n",
    "    qwks = [sum(score) / len(score) for score in qwks]\n",
    "\n",
    "    # save the results in a pandas dataframe\n",
    "    for essay_set in range(1, 9):\n",
    "        meta_data = essay_set_descriptions[essay_set - 1]\n",
    "        score_range = meta_data['score_ranges'][0]\n",
    "        filtered_df = df[df[\"essay_set\"] == essay_set]\n",
    "        calculated_qwk_filtered = qwks[essay_set - 1]\n",
    "\n",
    "        new_df_entry = {f\"Essay Set {essay_set}\": calculated_qwk_filtered, 'Average': sum(qwks) / len(qwks),\n",
    "                        'Incorrect Predictions': wrong_predictions, 'template': experiment['template'],\n",
    "                        'variant': experiment['variant'], 'prompt': experiment['prompt']}\n",
    "        df_data.append(new_df_entry)\n",
    "\n",
    "# create the dataframe and save it to a csv file\n",
    "df_data = pd.DataFrame(df_data)\n",
    "df_data.to_csv(\"./data/results/qwk_table.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "69053fae66a250fc"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Consistency Evaluation\n",
    "\n",
    "The experiments for the consistency evaluation can be conducted via:\n",
    "```bash\n",
    "python main.py --logging_data_path=\"./data/consistency/template${template}_${variant}/${prompt}.json\" --prompt=one_shot_prompt --dataset_split=dev --model_size=\"7b\" --model=\"mistral\" --prompt-template=1 --instruction-variant=1 --setting=one-shot --consistency\n",
    "```\n",
    "\n",
    "This command will generate the QWK scores for the one-shot prompt using the basic template and the original formulation variant. The variable `experiments` then has to be filled with all the experiments that have been conducted.\n",
    "\n",
    "The result of the following cell is a pandas dataframe that contains the MSE, MAE and cosine similarity for each essay set and the average over all essay sets. Using this dataframe, the results can be visualized and analyzed."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "73a8dc95eb2bb5da"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        'data_path': \"./data/consistency/template1_1/one_shot_prompt.json\",\n",
    "        'prompt': \"one_shot_prompt\",\n",
    "        'template': 1,\n",
    "        'variant': 1,\n",
    "    },\n",
    "]\n",
    "\n",
    "data = []\n",
    "score_ranges = [essay_set_descriptions[i]['score_ranges'][0][1] - essay_set_descriptions[i]['score_ranges'][0][0] for i\n",
    "                in range(8)]\n",
    "embedding_model = create_embedding()\n",
    "\n",
    "# embed all feedback texts first for a faster later comparison \n",
    "for i, experiment in tqdm(enumerate(experiments)):\n",
    "    with (open(experiment[\"data_path\"]), \"r\") as f:\n",
    "        data = json.load(f)\n",
    "\n",
    "    # group the essays by their essay_id that they were translated from using the backtranslation method \n",
    "    groups = {}\n",
    "    essay_to_essay_set = {}\n",
    "    for key in data.keys():\n",
    "        for essay in data[key]:\n",
    "            if essay['essay_id'] not in groups.keys():\n",
    "                groups[essay['essay_id']] = [essay]\n",
    "                essay_to_essay_set[essay['essay_id']] = int(key)\n",
    "            else:\n",
    "                groups[essay['essay_id']].append(essay)\n",
    "\n",
    "    score_sets = [[] for _ in range(8)]\n",
    "\n",
    "    for essay_id in groups:\n",
    "        essay_set = essay_to_essay_set[essay_id]\n",
    "        scoring_function = essay_set_descriptions[essay_set - 1]['full_score_fn']\n",
    "        try:\n",
    "            true_scores = [essay['correct_grade'] for essay in groups[essay_id]]\n",
    "            scores = [scoring_function(essay['parsed_output'], essay['parsed_output']) for essay in groups[essay_id]]\n",
    "\n",
    "            score_sets[essay_set - 1].append(scores)\n",
    "        except Exception as e:\n",
    "            pass\n",
    "\n",
    "            # calculate the MSE and MAE for each score_set with the majority vote as the predicted score\n",
    "    y = [[] for _ in range(8)]\n",
    "    y_hat = [[] for _ in range(8)]\n",
    "\n",
    "    for i, score_set in enumerate(score_sets):\n",
    "        for score in score_set:\n",
    "            y[i].extend(score)\n",
    "            y_hat[i].extend([max(set(score), key=score.count)] * len(score))\n",
    "\n",
    "    mse_normalized = [mean_squared_error(y[i], y_hat[i]) ** 0.5 / (score_ranges[i]) for i in range(8)]\n",
    "    mae_normalized = [mean_absolute_error(y[i], y_hat[i]) / score_ranges[i] for i in range(8)]\n",
    "\n",
    "    average_mae_normalized = np.mean(mse_normalized)\n",
    "    average_mse_normalized = np.mean(mae_normalized)\n",
    "\n",
    "    # calculate the cosine similarity\n",
    "    # embed all feedback texts first for a faster later comparison\n",
    "    clusters = {}\n",
    "    for key in data.keys():\n",
    "        embeddings = embedding_model.embed_documents([entry['output'] for entry in data[key]])\n",
    "        for i, essay in enumerate(data[key]):\n",
    "            if essay['essay_id'] not in clusters.keys():\n",
    "                clusters[essay['essay_id']] = [embeddings[i]]\n",
    "            else:\n",
    "                clusters[essay['essay_id']].append(embeddings[i])\n",
    "\n",
    "    # compare the all combinations of feedback texts \n",
    "    with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:\n",
    "        cluster_similarities = list(\n",
    "            tqdm(pool.imap(calculate_similarity_for_cluster_part, clusters.items()), total=len(clusters)))\n",
    "\n",
    "    average = np.mean([entry['average_cosine_similarity'] for entry in cluster_similarities])\n",
    "\n",
    "    data.append({\n",
    "        'template': experiment['template'],\n",
    "        'variant': experiment['variant'],\n",
    "        'prompt': experiment['prompt'],\n",
    "        'average_mse_normalized': str(round(average_mse_normalized, 3)),\n",
    "        'average_mae_normalized': str(round(average_mae_normalized, 3)),\n",
    "        'average_cosine_similarity': str(round(average, 3)),\n",
    "        **{f\"MSE ES {i + 1}\": str(round(mse_normalized[i], 3)) for i in range(8)},\n",
    "        **{f\"MAE ES {i + 1}\": str(round(mae_normalized[i], 3)) for i in range(8)},\n",
    "    })\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# save the df as a csv\n",
    "df.to_csv(\"../data/results/interconsistency.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "abfbc342e2e81f9f"
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Reference Points for Cosine Similarity \n",
    "\n",
    "We calculated two reference points for the cosine similarity to make the values more comparable. "
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "212f881a9c61d2b2"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "data_path = ''  # add correct path to the helpfulness results file (.json)\n",
    "number_of_samples = 30\n",
    "\n",
    "with open(data_path, \"r\") as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "samples = []\n",
    "for _ in range(number_of_samples):\n",
    "    fold = random.choice(list(data.keys()))\n",
    "    feedback = random.choice(data[fold])\n",
    "    samples.append(feedback)\n",
    "\n",
    "template = \"\"\"\n",
    "{model_prefix}\n",
    "Here is a text that needs to be reformulated. Don't change its meaning, but try to express it differently. \n",
    "'''\n",
    "{feedback}\n",
    "'''\n",
    "{model_suffix}\n",
    "\"\"\"\n",
    "\n",
    "prompt_template = PromptTemplate.from_template(template)\n",
    "\n",
    "\n",
    "class Config:\n",
    "    def __init__(self):\n",
    "        self.model = 'mistral'\n",
    "        self.temperature = 0\n",
    "        self.max_length = 4096\n",
    "\n",
    "\n",
    "config = Config()\n",
    "\n",
    "llm, prompt_template = load_model(config, prompt_template)\n",
    "\n",
    "# reformulate the feedback \n",
    "chain = prompt_template | llm\n",
    "for i in range(len(samples)):\n",
    "    samples[i]['reformulated_output'] = chain.invoke({'feedback': samples[i]['output']})\n",
    "\n",
    "\n",
    "embeddings_model = create_embedding()\n",
    "similarities_reformulated = []\n",
    "for sample in samples:\n",
    "    similarities_reformulated.append(compare_texts(sample['output'], sample['reformulated_output'], embeddings_model))\n",
    "\n",
    "similarities_original = []\n",
    "for i in range(len(samples)):\n",
    "    for j in range(i + 1, len(samples)):\n",
    "        similarities_original.append(compare_texts(samples[i]['output'], samples[j]['output'], embeddings_model))\n",
    "\n",
    "print(\n",
    "    f\"Reference point reformulated feedback: {np.mean([sim['cosine_similarity'] for sim in similarities_reformulated])}\")\n",
    "print(f\"Reference point random feedback: {np.mean([sim['cosine_similarity'] for sim in similarities_original])}\")"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b21be024b2ab546c"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Helpfulness Evaluation"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "411d53f4e1a3ef55"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Helpfulness Score Evaluation\n",
    "\n",
    "The experiments for the helpfulness score evaluation can be conducted via:\n",
    "```bash\n",
    "python helpfulness_evaluation.py --logging_data_path=\"./data/helpfulness/template1_1_mistral/one_shot_prompt.json\" --input_data=\"./data/qwk/template1_1/one_shot_prompt.json\" --prompt=one_shot_prompt --model_size=\"7b\" --model='mistral'\n",
    "```\n",
    "This command will generate the helpfulness scores for the one-shot prompt using the basic template and the original formulation variant. The variable `experiments` then has to be filled with all the experiments that have been conducted.\n",
    "\n",
    "The result of the following cell is a pandas dataframe that contains the helpfulness scores. Using this dataframe, the results can be visualized and analyzed."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "435d758c57afb062"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        'data_path': \"./data/helpfulness/template1_1_mistral/one_shot_prompt.json\",\n",
    "        'prompt': \"one_shot_prompt\",\n",
    "        'template': 1,\n",
    "        'variant': 1,\n",
    "        'model': 'mistral'\n",
    "    },\n",
    "]\n",
    "\n",
    "data = []\n",
    "for i, experiment in tqdm(enumerate(experiments)):\n",
    "    with (open(experiment[\"data_path\"]), \"r\") as f:\n",
    "        data = json.load(f)\n",
    "\n",
    "    helpfulness_scores = [int(entry['parsed_output']['score']) for entry in data if\n",
    "                          entry['parsed_output'] is not None and not isinstance(entry['parsed_output']['score'],\n",
    "                                                                                dict) and not isinstance(\n",
    "                              entry['parsed_output']['score'], list) and entry['parsed_output']['score'] is not None]\n",
    "\n",
    "    # filter out the values out of range \n",
    "    helpfulness_scores = [int(score) for score in helpfulness_scores if score in list(range(0, 11))]\n",
    "\n",
    "    mu = round(mean(helpfulness_scores), 2)\n",
    "    sigma = round(std(helpfulness_scores), 2)\n",
    "\n",
    "    data.append({\n",
    "        \"template\": experiment['template'],\n",
    "        \"variant\": experiment['variant'],\n",
    "        \"prompt\": experiment['prompt'],\n",
    "        \"model\": experiment['model'],\n",
    "        'Help Mean': mu,\n",
    "        'Help Std': sigma,\n",
    "    })\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "df.sort_values('Help Mean M', ascending=False)\n",
    "\n",
    "# save the df to a csv\n",
    "df.to_csv(\"./data/results/helpfulness.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "17723ee28905b80b"
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Verification of the Approach\n",
    "\n",
    "The approach was verified using the helpfulness verification dataset (`helpfulness_example_dataset.py`). The following cell contains the code for the verification of the approach.\n",
    "The helpfulness scores need to be saved previously using the following command for each model: \n",
    "```bash\n",
    "python helpfulness_evaluation.py --logging_data_path=\"./data/helpfulness/reference_mistral.json\" --model_size=\"7b\" --model=mistral --reference_dataset\n",
    "```"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "fb5ba49aed88a0b8"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open(\"./data/helpfulness/reference_mistral.json\", \"r\") as f:\n",
    "    mistral_data = json.load(f)\n",
    "\n",
    "with open(\"./data/helpfulness/reference_llama.json\", \"r\") as f:\n",
    "    llama_data = json.load(f)\n",
    "\n",
    "llama_gradings = [[] for _ in range(3)]\n",
    "mistral_gradings = [[] for _ in range(3)]\n",
    "\n",
    "mapping = {\n",
    "    NOT_HELPFUL: 0,\n",
    "    SEMI_HELPFUL: 1,\n",
    "    HELPFUL: 2\n",
    "}\n",
    "\n",
    "for llama, mistral, reference in zip(llama_data, mistral_data, samples):\n",
    "    assert llama['feedback'] == mistral['feedback'] == reference['feedback']\n",
    "    llama_gradings[mapping[reference['label']]].append(llama['parsed_output']['score'])\n",
    "    mistral_gradings[mapping[reference['label']]].append(mistral['parsed_output']['score'])\n",
    "    print(llama['parsed_output']['score'], \"\\t\", mistral['parsed_output']['score'], \"\\t\", reference['label'])\n",
    "\n",
    "data = []\n",
    "labels = ['not-helpful', 'semi-helpful', 'helpful']\n",
    "for i in range(3):\n",
    "    for score in llama_gradings[i]:\n",
    "        data.append({'label': labels[i], 'score': score, 'model': 'llama'})\n",
    "    for score in mistral_gradings[i]:\n",
    "        data.append({'label': labels[i], 'score': score, 'model': 'mistral'})\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Calculate the mean and standard deviation for each group\n",
    "df = df.groupby(['label', 'model']).agg(['mean', 'std']).reset_index()\n",
    "print(df)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "effce971e3aa3199"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Pairwise Comparison Evaluation\n",
    "\n",
    "The experiment for the pairwise comparison evaluation can be conducted via:\n",
    "```bash\n",
    "python elo_ranking.py --logging_data_path=\"./data/elo/matches\" --model_size=\"7b\" --model=\"mistral\"\n",
    "```\n",
    "This command will use the dataset in `.data/datasets/elo` to make all pairwise comparisons between the feedback texts in the dataset. \n",
    "\n",
    "The result of the following cell is a pandas dataframe that contains the contains the wins and losses for each prompt that is contained in the dataset. Using this dataframe, the results can be visualized and analyzed."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "e7d2ee1687d86f20"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "dataset = load_from_disk(\"../data/datasets/elo_ranking\")\n",
    "\n",
    "with open(f\"./data/elo/matches.json\", \"r\") as f:\n",
    "    results = json.load(f)\n",
    "\n",
    "all_experiments = set([entry['experiment'] for entry in dataset])\n",
    "\n",
    "wins = {exp: 0 for exp in all_experiments}\n",
    "losses = {exp: 0 for exp in all_experiments}\n",
    "wins_against = {\n",
    "    exp: {exp2: 0 for exp2 in all_experiments}\n",
    "    for exp in all_experiments\n",
    "}\n",
    "\n",
    "for match in tqdm(results):\n",
    "    # skip if the LLM did not produce a parsed output\n",
    "    if match['parsed_output'] is None:\n",
    "        continue\n",
    "    if match['parsed_output']['preferred_feedback'] == 1:\n",
    "        wins[dataset[match['index1']]['experiment']] += 1\n",
    "        losses[dataset[match['index2']]['experiment']] += 1\n",
    "        wins_against[dataset[match['index1']]['experiment']][dataset[match['index2']]['experiment']] += 1\n",
    "    elif match['parsed_output']['preferred_feedback'] == 2:\n",
    "        wins[dataset[match['index2']]['experiment']] += 1\n",
    "        losses[dataset[match['index1']]['experiment']] += 1\n",
    "        wins_against[dataset[match['index2']]['experiment']][dataset[match['index1']]['experiment']] += 1\n",
    "        \n",
    "# create the dataframe \n",
    "data = {\n",
    "    \"experiment\": [],\n",
    "    \"wins\": [],\n",
    "    \"losses\": [],\n",
    "}\n",
    "\n",
    "data.update({\n",
    "    f\"wins_against_{exp}\": []\n",
    "    for exp in all_experiments\n",
    "})\n",
    "\n",
    "for exp in all_experiments:\n",
    "    data['experiment'].append(exp)\n",
    "    data['wins'].append(wins[exp])\n",
    "    data['losses'].append(losses[exp])\n",
    "    for exp2 in sorted(all_experiments):\n",
    "        data[f\"wins_against_{exp}\"].append(wins_against[exp][exp2])\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# save the dataframe\n",
    "df.to_csv(\"./data/results/elo_ranking.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "673e1d55af1c950f"
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Verification of the Approach\n",
    "\n",
    "The approach was verified using a reference dataset (`helpfulness_example_dataset.py`). The following cell contains the code for the verification of the approach."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "fa8904e7b6e6f785"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "template = \"\"\"\n",
    "{model_prefix}\n",
    "# Task \n",
    "You are given now given two feedbacks and should decide which one is better. If the first feedback is better, type 1. If the second feedback is better, type 2. You have to decide for one feedback based on the following criteria:\n",
    "A good feedback is a feedback that indicates what are the errors, why the errors are errors and makes concrete suggestions on how to fix them. \n",
    "\n",
    "#### Feedback 1\n",
    "```{feedback1}```\n",
    "\n",
    "#### Feedback 2\n",
    "```{feedback2}```\n",
    "\n",
    "#### Which feedback is better?\n",
    "Type 1 if the first feedback is better, type 2 if the second feedback is better.\n",
    "\n",
    "{format_instructions}\n",
    "\n",
    "{model_suffix}\n",
    "\"\"\"\n",
    "\n",
    "prompt_template = PromptTemplate(template=template, input_variables=[\"model_prefix\", \"model_suffix\", \"feedback1\",\n",
    "                                                                     \"feedback2\", \"format_instructions\"])\n",
    "\n",
    "\n",
    "class Config:\n",
    "    def __init__(self):\n",
    "        self.model = 'mistral'\n",
    "        self.max_length = 4096\n",
    "        self.temperature = 0\n",
    "\n",
    "\n",
    "config = Config()\n",
    "llm, prompt_template = load_model(config, prompt_template)\n",
    "output_parser = StructuredOutputParser.from_response_schemas(response_schemas=[\n",
    "    ResponseSchema(name=\"preferred_feedback\", description=\"the preferred feedback (1 for feedback 1, 2 for feedback 2\",\n",
    "                   type=\"int\")\n",
    "])\n",
    "\n",
    "all_pairs = []\n",
    "for i in range(len(samples)):\n",
    "    for j in range(i + 1, len(samples)):\n",
    "        all_pairs.append({\"feedback1\": samples[i]['feedback'], \"feedback2\": samples[j]['feedback'], \"i\": i, \"j\": j})\n",
    "results = collect_llm_output(llm, all_pairs, output_parser, prompt_template)\n",
    "\n",
    "# get the number of wins and losses for each category\n",
    "wins = {\n",
    "    \"helpful\": 0,\n",
    "    \"semi-helpful\": 0,\n",
    "    \"not-helpful\": 0\n",
    "}\n",
    "losses = {\n",
    "    \"helpful\": 0,\n",
    "    \"semi-helpful\": 0,\n",
    "    \"not-helpful\": 0\n",
    "}\n",
    "wins_against = {\n",
    "    \"helpful\": {\"helpful\": 0, \"semi-helpful\": 0, \"not-helpful\": 0},\n",
    "    \"semi-helpful\": {\"helpful\": 0, \"semi-helpful\": 0, \"not-helpful\": 0},\n",
    "    \"not-helpful\": {\"helpful\": 0, \"semi-helpful\": 0, \"not-helpful\": 0}\n",
    "}\n",
    "\n",
    "for result in results:\n",
    "    # skip if the LLM did not produce a parsed output\n",
    "    if result['parsed_output'] is None:\n",
    "        continue\n",
    "    if result['parsed_output']['preferred_feedback'] == 1:\n",
    "        winner = result['i']\n",
    "        looser = result['j']\n",
    "    elif result['parsed_output']['preferred_feedback'] == 2:\n",
    "        winner = result['j']\n",
    "        looser = result['i']\n",
    "    else:\n",
    "        continue\n",
    "\n",
    "    wins[samples[winner]['label']] += 1\n",
    "    losses[samples[looser]['label']] += 1\n",
    "    wins_against[samples[winner]['label']][samples[looser]['label']] += 1\n",
    "\n",
    "data = {\n",
    "    \"label\": [],\n",
    "    \"wins\": [],\n",
    "    \"losses\": [],\n",
    "    \"wins_against_helpful\": [],\n",
    "    \"wins_against_semi_helpful\": [],\n",
    "    \"wins_against_not_helpful\": []\n",
    "}\n",
    "\n",
    "for label in wins:\n",
    "    data['label'].append(label)\n",
    "    data['wins'].append(wins[label])\n",
    "    data['losses'].append(losses[label])\n",
    "    data['wins_against_helpful'].append(wins_against[label][\"helpful\"])\n",
    "    data['wins_against_semi_helpful'].append(wins_against[label][\"semi-helpful\"])\n",
    "    data['wins_against_not_helpful'].append(wins_against[label][\"not-helpful\"])\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "print(df)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "cf1b4492d2f7a764"
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Manual Annotation Study\n",
    "\n",
    "For the MAS, three things were done: selecting feedback texts which are included, calculate IAA, calculate agreement with automatic evaluations, and calculate averages for each prompt. \n",
    "\n",
    "#### Select Feedback Texts"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "a7299bfb77c58fc2"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df = pd.read_excel('./data/training_set_rel3.xlsx', index_col='essay_id')\n",
    "\n",
    "dataset = load_from_disk('./data/datasets/consistency')\n",
    "\n",
    "# select all essays from the dataset that belong to essay set 4 \n",
    "essay_ids_from_essay_set_4 = set()\n",
    "for entry in dataset:\n",
    "    if entry['essay_set'] == 4:\n",
    "        essay_ids_from_essay_set_4.add(entry['essay_id'])\n",
    "\n",
    "# put the essay ids into buckets based on their grade\n",
    "buckets = [[] for _ in range(4)]\n",
    "for id in essay_ids_from_essay_set_4:\n",
    "    grade = df.loc[id]['domain1_score']\n",
    "    buckets[int(grade)].append(id)\n",
    "\n",
    "random.seed(42)\n",
    "\n",
    "# randomly select two essays from each bucket for a total of 8 essays\n",
    "selected_essays = []\n",
    "for bucket in buckets:\n",
    "    selected_essays.extend(random.sample(bucket, 2))\n",
    "\n",
    "print(selected_essays)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "28ef165a913df195"
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Preparing the data \n",
    "\n",
    "First, the data has to be prepared to get it into a usable format. We used a pandas dataframe here."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "f7f058d684e5bbb6"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df_ref = pd.read_csv('../data/mas_results.csv')\n",
    "\n",
    "values = []\n",
    "graded_combinations = {token: set() for token in df_ref['token']}\n",
    "annotators = list(graded_combinations.keys())\n",
    "\n",
    "for i, row in df_ref.iterrows():\n",
    "    token = row['token']\n",
    "    for column in df_ref.columns:\n",
    "        if \"SQ\" not in column:\n",
    "            continue\n",
    "\n",
    "        if row[column] is None or not isinstance(row[column], str):\n",
    "            continue\n",
    "\n",
    "        question = int(column[-2])\n",
    "        prompt = column.split(\"0\")[0]\n",
    "        essay = int(column[-9:-7])\n",
    "\n",
    "        values.append({\n",
    "            \"prompt\": prompt,\n",
    "            \"essay\": essay,\n",
    "            \"question\": question,\n",
    "            token: int(row[column][0])\n",
    "        })\n",
    "        graded_combinations[token].add((prompt, essay))\n",
    "\n",
    "df = pd.DataFrame(values)\n",
    "\n",
    "# create a ranking for the feedback texts for each annotator (limited to the texts that this annotator has graded)\n",
    "annotator_rankings_dict = {}\n",
    "for annotator in annotators:\n",
    "    annotator_rankings_dict[annotator] = {}\n",
    "    for prompt, essay in graded_combinations[annotator]:\n",
    "        overall_score = \\\n",
    "        df[(df['prompt'] == prompt) & (df['essay'] == essay) & (df['question'] == 5)][annotator].dropna().iloc[0]\n",
    "        tiebreaker_score = df[(df['prompt'] == prompt) & (df['essay'] == essay) & (df['question'] != 5)][\n",
    "            annotator].dropna().sum()\n",
    "        annotator_rankings_dict[annotator][(prompt, essay)] = (overall_score, tiebreaker_score)\n",
    "\n",
    "    annotator_rankings_dict[annotator] = sorted(annotator_rankings_dict[annotator].items(),\n",
    "                                                key=lambda x: (x[1][0], x[1][1]), reverse=True)\n",
    "annotator_rankings = sorted(annotator_rankings_dict.items(), key=lambda x: x[0])"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "437d521bfd62aa08"
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Calculate IAA\n",
    "\n",
    "We used the Krippendorff's alpha to calculate the IAA. The following cell contains the code for the calculation of the IAA."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "382f65f7acfa0ba1"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# calculate the krippendorff alpha for all annotators that start with the same letter\n",
    "annotator_groups = [['Affen', 'Alexa', 'Andre'], ['Bohle', 'Bombe', 'Bulli'], ['Cande', 'Cesar', 'Cloth'],\n",
    "                    ['Demut', 'Dobby', 'Doria']]\n",
    "\n",
    "group_agreement = {}\n",
    "for group in annotator_groups:\n",
    "    annotator_values = [annotator_rankings_dict[annotator] for annotator in group]\n",
    "    annotator_values = [[item[1][0] for item in annotator] for annotator in annotator_values]\n",
    "\n",
    "    matrix = np.array(annotator_values)\n",
    "\n",
    "    group_agreement[group[0][0]] = krippendorff.alpha(reliability_data=matrix, level_of_measurement='ordinal')\n",
    "\n",
    "    print(group, round(krippendorff.alpha(reliability_data=matrix, level_of_measurement='ordinal'), 2))"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "548734cb779448e6"
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Calculate Agreement with Automatic Evaluations\n",
    "\n",
    "We used the Kendall's Tau to calculate the agreement with the automatic evaluations. The following cell contains the code for the calculation of the agreement with the automatic evaluations."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "beef8cd9df71538d"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "dataset_df = pd.read_excel(\"../data/training_set_rel3.xlsx\", index_col='essay_id')\n",
    "\n",
    "essay_ids = {\n",
    "    'dcot': [9098, 9833, 9486, 9512, 9399, 9047, 10292, 8995],\n",
    "    'fo': [9098, 9486, 9833, 9512, 9399, 8995, 9047, 10292],\n",
    "    'ff': [9098, 9486, 9833, 9512, 9399, 8995, 10292, 9047]\n",
    "}\n",
    "\n",
    "essays = {\n",
    "    'dcot': dataset_df.loc[essay_ids['dcot']].essay.tolist(),\n",
    "    'fo': dataset_df.loc[essay_ids['fo']].essay.tolist(),\n",
    "    'ff': dataset_df.loc[essay_ids['ff']].essay.tolist()\n",
    "}\n",
    "\n",
    "# get the helpfulness scores for the mistral model\n",
    "pathes = {\n",
    "    'dcot': '',  # add correct path to the helpfulness results file for the mistral model (.json)\n",
    "    'fo': '',  # add correct path to the helpfulness results file for the mistral model (.json)\n",
    "    'ff': '',  # add correct path to the helpfulness results file for the mistral model (.json)\n",
    "}\n",
    "\n",
    "mistral_gradings = {'dcot': [None for _ in range(8)], 'fo': [None for _ in range(8)], 'ff': [None for _ in range(8)]}\n",
    "for prompt in ['dcot', 'fo', 'ff']:\n",
    "    with open(pathes[prompt], \"r\") as f:\n",
    "        results = json.load(f)\n",
    "    for essay in data:\n",
    "        if essay['essay'] in essays[prompt]:\n",
    "            mistral_gradings['fo'][essays[prompt].index(essay['essay'])] = essay['parsed_output']['score']\n",
    "\n",
    "        # get the helpfulness scores for the llama model \n",
    "pathes = {\n",
    "    'dcot': '',  # add correct path to the helpfulness results file for the llama model (.json)\n",
    "    'fo': '',  # add correct path to the helpfulness results file for the llama model (.json)\n",
    "    'ff': '',  # add correct path to the helpfulness results file for the llama model (.json)\n",
    "}\n",
    "\n",
    "llama_gradings = {'dcot': [None for _ in range(8)], 'fo': [None for _ in range(8)], 'ff': [None for _ in range(8)]}\n",
    "for prompt in ['dcot', 'fo', 'ff']:\n",
    "    with open(pathes[prompt], \"r\") as f:\n",
    "        results = json.load(f)\n",
    "    for essay in data:\n",
    "        if essay['essay'] in essays[prompt]:\n",
    "            llama_gradings['fo'][essays[prompt].index(essay['essay'])] = essay['parsed_output']['score']\n",
    "\n",
    "# make a ranking out of the mistral and llama gradings for all essays \n",
    "mistral_ranking = [(mistral_gradings[instruction][i], instruction, i) for instruction in mistral_gradings for i in\n",
    "                   range(8)]\n",
    "llama_ranking = [(llama_gradings[instruction][i], instruction, i) for instruction in llama_gradings for i in range(8)]\n",
    "\n",
    "mistral_ranking = sorted(mistral_ranking, key=lambda x: x[0], reverse=True)\n",
    "llama_ranking = sorted(llama_ranking, key=lambda x: x[0], reverse=True)\n",
    "\n",
    "# make a combined ranking out of all the human annotations \n",
    "feedbacks = {}\n",
    "for annotator, annotator_ranking in annotator_rankings_dict.items():\n",
    "    for (feedback, score) in annotator_ranking:\n",
    "        if feedback not in feedbacks:\n",
    "            feedbacks[feedback] = 0\n",
    "        feedbacks[feedback] += score[0]\n",
    "combined_human_ranking = sorted(feedbacks.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "# calculate the kendall tau between the rankings by giving all feedbacks a ranking and then sorting them alphabetically \n",
    "mistral_ranking_values = [None for _ in range(24)]\n",
    "current_rank = 1\n",
    "for i in range(10, 0, -1):\n",
    "    found = False\n",
    "    for j in range(24):\n",
    "        if mistral_ranking[j][0] == i:\n",
    "            mistral_ranking_values[j] = (current_rank, (mistral_ranking[j][1], mistral_ranking[j][2]))\n",
    "            found = True\n",
    "    if found:\n",
    "        current_rank += 1\n",
    "\n",
    "llama_ranking_values = [None for _ in range(24)]\n",
    "current_rank = 1\n",
    "for i in range(10, 0, -1):\n",
    "    found = False\n",
    "    for j in range(24):\n",
    "        if llama_ranking[j][0] == i:\n",
    "            llama_ranking_values[j] = (current_rank, (llama_ranking[j][1], llama_ranking[j][2]))\n",
    "            found = True\n",
    "    if found:\n",
    "        current_rank += 1\n",
    "\n",
    "combined_human_ranking_values = [None for _ in range(24)]\n",
    "current_rank = 1\n",
    "for i in range(24):\n",
    "    combined_human_ranking_values[i] = (\n",
    "    current_rank, (combined_human_ranking[i][0][0].lower(), combined_human_ranking[i][0][1]))\n",
    "    current_rank += 1\n",
    "\n",
    "mistral_ranking_values = sorted(mistral_ranking_values, key=lambda x: x[1])\n",
    "llama_ranking_values = sorted(llama_ranking_values, key=lambda x: x[1])\n",
    "combined_human_ranking_values = sorted(combined_human_ranking_values, key=lambda x: x[1])\n",
    "\n",
    "print(\n",
    "    f\"Kendalls Tau Mistral and Human: {kendalltau([i[0] for i in mistral_ranking_values], [i[0] for i in combined_human_ranking_values]).correlation}\")\n",
    "print(\n",
    "    f\"Kendalls Tau Llama and Human: {kendalltau([i[0] for i in llama_ranking_values], [i[0] for i in combined_human_ranking_values]).correlation}\")\n",
    "\n",
    "# get the number of wins and losses and wins against for each list of items \n",
    "wins = {}\n",
    "losses = {}\n",
    "wins_against = {}\n",
    "\n",
    "indices = [i for i in range(len(dataset)) if dataset[i]['essay_id'] in essay_ids['dcot'] and (\"dcot\" in dataset[i]['experiment'] or \"feedback_only\" in dataset[i]['experiment'] or \"ff\" in dataset[i]['experiment'])]\n",
    "\n",
    "with open(f\"./data/elo/matches.json\", \"r\") as f:\n",
    "    matches = json.load(f)\n",
    "\n",
    "for match in tqdm(matches): \n",
    "    if match['parsed_output'] is None: \n",
    "        continue\n",
    "    if match['index1'] not in indices or match['index2'] not in indices: \n",
    "        continue\n",
    "    if match['parsed_output']['preferred_feedback'] == 1: \n",
    "        if match['index1'] not in wins: \n",
    "            wins[match['index1']] = 0\n",
    "        wins[match['index1']] += 1\n",
    "        if match['index2'] not in losses: \n",
    "            losses[match['index2']] = 0\n",
    "        losses[match['index2']] += 1\n",
    "        if match['index1'] not in wins_against: \n",
    "            wins_against[match['index1']] = {}\n",
    "        if match['index2'] not in wins_against[match['index1']]: \n",
    "            wins_against[match['index1']][match['index2']] = 0\n",
    "        wins_against[match['index1']][match['index2']] += 1\n",
    "    elif match['parsed_output']['preferred_feedback'] == 2:\n",
    "        if match['index2'] not in wins: \n",
    "            wins[match['index2']] = 0\n",
    "        wins[match['index2']] += 1\n",
    "        if match['index1'] not in losses: \n",
    "            losses[match['index1']] = 0\n",
    "        losses[match['index1']] += 1\n",
    "        if match['index2'] not in wins_against: \n",
    "            wins_against[match['index2']] = {}\n",
    "        if match['index1'] not in wins_against[match['index2']]: \n",
    "            wins_against[match['index2']][match['index1']] = 0\n",
    "        wins_against[match['index2']][match['index1']] += 1\n",
    "        \n",
    "# order the items so that the essay ids are in the same order as the essay_ids_*\n",
    "items_dcot = [(dataset[i], i) for i in range(len(dataset)) if \"dcot\" in dataset[i]['experiment'] and dataset[i]['essay_id'] in essay_ids['dcot']]\n",
    "items_fo = [(dataset[i], i) for i in range(len(dataset)) if (\"feedback_only\") in dataset[i]['experiment'] and dataset[i]['essay_id'] in essay_ids['dcot']]\n",
    "items_ff = [(dataset[i], i) for i in range(len(dataset)) if \"ff\" in dataset[i]['experiment'] and dataset[i]['essay_id'] in essay_ids['dcot']]\n",
    "items_dcot = sorted(items_dcot, key=lambda x: essay_ids['dcot'].index(x[0]['essay_id']))\n",
    "items_fo = sorted(items_fo, key=lambda x: essay_ids['fo'].index(x[0]['essay_id']))\n",
    "items_ff = sorted(items_ff, key=lambda x: essay_ids['ff'].index(x[0]['essay_id']))\n",
    "\n",
    "ranking_values = []\n",
    "for item in items_dcot:\n",
    "    id = item[0]['essay_id']\n",
    "    ranking_values.append(((\"dcot\", essay_ids['dcot'].index(id)), (wins[item[1]], losses[item[1]])))\n",
    "    \n",
    "for item in items_ff:\n",
    "    id = item[0]['essay_id']\n",
    "    ranking_values.append(((\"ff\", essay_ids['ff'].index(id)), (wins[item[1]], losses[item[1]])))\n",
    "    \n",
    "for item in items_fo:\n",
    "    id = item[0]['essay_id']\n",
    "    ranking_values.append(((\"fo\", essay_ids['fo'].index(id)), (wins[item[1]], losses[item[1]])))\n",
    "    \n",
    "mistral_ranking_values = [None for _ in range(24)]\n",
    "current_rank = 1\n",
    "for i in range(30, 0, -1):\n",
    "    found = False\n",
    "    for j in range(24):\n",
    "        if ranking_values[j][1][0] == i:\n",
    "            mistral_ranking_values[j] = (current_rank, ranking_values[j][0])\n",
    "            found = True\n",
    "    if found:\n",
    "        current_rank += 1\n",
    "        \n",
    "print(kendalltau([x[0] for x in combined_human_ranking_values], [x[0] for x in mistral_ranking_values]))"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "5721efc150a66e5f"
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Calculate Averages for Each Prompt\n",
    "\n",
    "The following cell contains the code for the calculation of the average response for each prompt."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "1debde044ef3d430"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# sum up the scores for each prompt\n",
    "prompt_scores = {'dCoT': [0 for _ in range(5)], 'FO': [0 for _ in range(5)], 'FF': [0 for _ in range(5)]}\n",
    "\n",
    "for i, row in df.iterrows():\n",
    "    prompt = row['prompt']\n",
    "    question = row['question']\n",
    "    score = row.dropna().iloc[3]\n",
    "\n",
    "    prompt_scores[prompt][question - 1] += score\n",
    "\n",
    "# take the average of the scores for each prompt\n",
    "for prompt in prompt_scores:\n",
    "    prompt_scores[prompt] = [round(score / 24, 2) for score in prompt_scores[prompt]]\n",
    "\n",
    "print(prompt_scores)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c43823db442c850b"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
